# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html


# useful for handling different item types with a single interface
from itemadapter import ItemAdapter
import pymysql
import pymysql.cursors

class BiguoquestionsPipeline:
    def process_item(self, item, spider):
        return item

class MysqlPipline(object):
    def __init__(self):
        self.conn = pymysql.connect('localhost', 'root', '123456', 'biguo_spider')
        self.cursor = self.conn.cursor()

    def process_item(self, item, spider):
        insert_sql = """
            insert into biguo_questions(course_name, types ,type_num, type_urls) VALUES (%s, %s, %s, %s)
        """
        # 同步化地插入数据库操作，当数据量大的时候，会形成堵塞
        self.cursor.execute(insert_sql,
                            (item["course_name"], item["types"], item["type_num"], item["type_urls"]))
        self.conn.commit()

    def close_spider(self, spider):
        # 关闭游标和连接
        self.cursor.close()
        self.conn.close()

